package org.fujene.term;

import java.io.CharArrayWriter;
import java.io.IOException;

import org.fujene.structure.Term;
import org.fujene.structure.TermList;

public class WordBigramChopper implements Chopper {
	private static final String stopWord = " 　,、.。<＜>＞!！\"”'’(（)）;；:：[「]」{｛}｝";

	private static final String[] exceptWord = {"a", "am", "an", "and", "are", "as", "at", 
		"be", "by", "can", "for", "in", "into", "is", "may", "must", "of", "on", "onto", "or",
		"this", "that", "the", "upon", "with", "without"};

	private void addStr(TermList strset, String str){
		for (int i = 0; i < exceptWord.length; i++) {
			if(str.equals(exceptWord[i]))
				return;
		}
		// 文字列の統一
		str = Escaper.convertSJISToMS932(str);
		
		// 半角カナ-> 全角カナ
		str = Optimizer.HanKanaToZen(str);
		
		// 全角アルファベット -> 半角
		str = Optimizer.ZenAlphaToHan(str);
		
		// アンエスケープ処理
		str = Escaper.unescapeEntityRef(str);
		
		// 半角記号 -> 全角記号
		str = Optimizer.HanSymbolToZen(str);
		
		str = str.toLowerCase();

//		System.out.println("Chop: " + str);
		strset.add(new Term(str));
	}
	
	@Override
	public TermList chop(String content) {
		TermList result = new TermList();

		char[] contchar = content.toCharArray();
		char[] surrogatePair = new char[2];
		char prevKanji = 0x0000;
		CharArrayWriter writer = new CharArrayWriter();
		// Rule: 数字列・英単語は1語とみなす
		for (int i = 0; i < contchar.length; i++) {
			if(stopWord.indexOf(contchar[i]) != -1){
				// ストップワード ... これまでの分を追加
				if(prevKanji != 0x0000){
					addStr(result, Character.toString(prevKanji));
					prevKanji = 0x0000;
				}
				addStr(result, writer.toString());
				writer.reset();
			}else if(contchar[i] >= 0x0021 && contchar[i] <= 0x007E){
				// ASCII文字
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				writer.append(contchar[i]);
			}else if(contchar[i] >= 0x00A1 && contchar[i] <= 0x2E31){
				// 各言語・記号
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				prevKanji = 0x0000;
				writer.append(contchar[i]);
			}else if(contchar[i] >= 0x3000 && contchar[i] <= 0x3040){
				// 各言語・記号
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				writer.append(contchar[i]);
			}else if(contchar[i] >= 0x3105 && contchar[i] <= 0x33FF){
				// 各言語・記号
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				writer.append(contchar[i]);
			}else if(contchar[i] >= 0xA500 && contchar[i] <= 0xABF9){
				// 各言語・記号
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				writer.append(contchar[i]);
			}else if(contchar[i] >= 0xD800 && contchar[i] <= 0xDBFF){
				// 上位サロゲート
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				surrogatePair[0] = contchar[i];
			}else if(contchar[i] >= 0xDC00 && contchar[i] <= 0xDFFF){
				// 下位サロゲート
				prevKanji = 0x0000;
				if(surrogatePair[0] >= 0xD800 && surrogatePair[0] <= 0xDBFF){
					surrogatePair[1] = contchar[i];
					try {
						writer.write(surrogatePair);
					} catch (IOException ex) {
					}
				}
			}else if(contchar[i] >= 0xFE00 && contchar[i] <= 0xFE0F){
				// 記号
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				writer.append(contchar[i]);
			}else if(contchar[i] >= 0xFFE0 && contchar[i] <= 0xFFFF){
				// 記号
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				writer.append(contchar[i]);
			}else if(contchar[i] >= 0xE000 && contchar[i] <= 0xF8FF){
				// 私用領域 -> 無視
				if(prevKanji != 0x0000){
					addStr(result, Character.toString(prevKanji));
					prevKanji = 0x0000;
				}
				addStr(result, writer.toString());
				writer.reset();
			}else{
				// bi-gram対象
				if(writer.size() != 0 && prevKanji == 0x0000){
					// 今までの分を追加
					addStr(result, writer.toString());
					writer.reset();
					// 1文字目追加
					addStr(result, Character.toString(contchar[i]));
				}else if(prevKanji != 0x0000){
					// 2文字ペア追加
					writer.append(contchar[i]);
					addStr(result, writer.toString());
					writer.reset();
				}else if(writer.size() == 0 && prevKanji == 0x0000){
					// 1文字目追加
					addStr(result, Character.toString(contchar[i]));
				}
				prevKanji = contchar[i];
				writer.append(contchar[i]);
			}
			
			if (i == contchar.length - 1) {
				// 残ったストリームを追加
				if (writer.size() != 0)
					addStr(result, writer.toString());
			}
		}

		return result;
	}
}
